package com.cold.demo.jsoup;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.*;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

/**
 * 爬取 aspose 的所有jar包
 * Created by zyc(378661553@qq.com) on 2021/3/31 16:40
 */
public class JsoupAsposeJar {

    final static String          baseUrl         = "https://repository.aspose.com/repo/";
    final static String          basePath        = "D:/Environment/localRepository-aspose";
    final static ExecutorService executorService = Executors.newCachedThreadPool();

    public static void main(String[] args) throws Exception {
//        writeUrl();
        downUrl();
    }

    private static void downUrl() throws Exception {
        BufferedReader bufferedReader = new BufferedReader(new FileReader(new File("C:\\Users\\zfsw\\Desktop\\fsdownload\\1.txt")));

        List<String> lines = new LinkedList<>();
        ExecutorService[] threads = new ExecutorService[Runtime.getRuntime().availableProcessors()];
        for (int i = 0; i < Runtime.getRuntime().availableProcessors(); i++) {
            threads[i] = Executors.newSingleThreadExecutor();
        }

        while (true){
            String         readLine       = bufferedReader.readLine();
            if (readLine==null){
                break;
            }
        }
        int i = 0;
        for (String readLine : lines) {

            i++;



            lines.add(readLine);
            String         url            = readLine.split("\t\t\t")[0];
            String         filePath       = readLine.split("\t\t\t")[1];
            InputStream    inputStream    = new URL(url).openStream();
            FileOutputStream fos = new FileOutputStream(filePath);
            byte[]         bytes          = new byte[4096];
            int len;
            while ((len = inputStream.read(bytes))!=-1){
                fos.write(bytes,0,len);
            }
            fos.flush();
            fos.close();
        }
    }

    private static void writeUrl() throws Exception {
        FileOutputStream fos       = new FileOutputStream(new File("C:\\Users\\zfsw\\Desktop\\fsdownload\\1.txt"));
        long             startTime = System.currentTimeMillis();
        try {
            download("https://repository.aspose.com/repo/com/aspose/", basePath, fos);
        } finally {
            System.out.println("本次下载完毕，耗时：" + (System.currentTimeMillis() - startTime) + "ms");
        }
    }

    static List<String> dwSuffix = Arrays.asList("jar", "pom", "xml", "zip", "aar", "apk");


    private static void download(String baseUrl, String basePath, OutputStream os) throws Exception {
        Document parse = Jsoup.parse(new URL(baseUrl), 30_000);
        Element  pre   = parse.getElementsByTag("pre").get(1);
        Elements as    = pre.getElementsByTag("a");
        for (Element a : as) {
            // https://repository.aspose.com/repo/com/aspose/aspose-words/21.2/aspose-words-21.2-jdk17.jar
            String href = a.attr("href");
            if (href.equals("../")) {
                continue;
            }
            if (href.lastIndexOf(".") != -1) {
                if (dwSuffix.contains(href.substring(href.lastIndexOf(".") + 1).toLowerCase())) {
                    /*Runnable runnable = () -> {
                        try {
                            String      url      = baseUrl + href;
                            InputStream in       = new URL(url).openStream();
                            String      filePath = basePath + url.replace(JsoupAsposeJar.baseUrl, "/");
                            File        file     = new File(filePath);
                            if (!file.getParentFile().exists()) {
                                file.getParentFile().mkdirs();
                            }
                            if (!file.exists()) {
                                file.createNewFile();
                            }
                            FileOutputStream out = new FileOutputStream(file);

                            byte[] buffer    = new byte[1024];
                            int    bytesRead = -1;
                            while ((bytesRead = in.read(buffer)) != -1) {
                                out.write(buffer, 0, bytesRead);
                            }
                            out.flush();
                            System.out.println("下载 " + url + " 成功，本地路径为： " + filePath);
                        } catch (Exception e) {
                            e.printStackTrace();
                        }
                    };
                    executorService.execute(runnable);*/

                    String url      = baseUrl + href;
                    String filePath = basePath + url.replace(JsoupAsposeJar.baseUrl, "/");

                    os.write(url.getBytes());
                    os.write("\t\t\t".getBytes());
                    os.write(filePath.getBytes());
                    os.write("\n".getBytes());
                    continue;
                }
            }
            download(baseUrl + href, basePath, os);
        }

    }
}
